import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


public class CrawlCNNArticles {

	public static void main(String args[]){
	
		CrawlCNNArticles cnn = new CrawlCNNArticles();
		cnn.crawlCNNArticles();
	}
	
	private String folderName ="C:\\Users\\lakshman\\workspace\\cse6339-faceted-interface\\News Articles\\CNN Crime News\\";
	private String processFileDirectory="C:\\Users\\lakshman\\workspace\\cse6339-faceted-interface\\Processed Articles\\Processed CNN Crime News";
	private FileIO file = new FileIO();
	
	private void crawlCNNArticles()
	{
		//cnn archive
		String url1 = "http://www.cnn.com/CRIME/archive/index.html";
		ArrayList<String> urls = new ArrayList<String>();
		urls.add(url1);
		for(String url : urls)
		{
			String retrievedText = downloadPage(url);
			if(!retrievedText.isEmpty()){
		        ArrayList<String> links = retrieveCNNLinks(retrievedText);
		        int articleCnt = 1;
		        if(links != null)
		        {
			        for(String link : links)
			        {
			        	System.out.println(link);
			        	String newsContent =  downloadPage(link);
			        	String title =downloadCNNArticleTitle(newsContent);
			        	String content = downloadArticleBody(newsContent);
			        		        	
/*			        	if(!content.isEmpty()){
				        	String article=title +"\n"+content;
				        	//System.out.println(title +"\n"+content);
				        	String filename = folderName + title + articleCnt + ".txt";
				        	file.writeToFile(article, filename);
				        	System.out.println(articleCnt);
				        	articleCnt++;
			        	}
*/			        
			        	break;
			        }
		        }
		        else
		        {
		        	System.out.println("ERROR: No links found in page: " + url);
		        }
			}
		}
	}
	
	private ArrayList<String> retrieveCNNLinks(String fullText)
	{
	
		if(fullText.isEmpty())
			System.out.println("No text in the webpage.");
		ArrayList<String> links = new ArrayList<String>();
		//Pattern p = Pattern.compile("<li><a\\s+href\\s*=\\s*\"?(.*?)[\"|>]?(.*?)</a></li></ul>", Pattern.CASE_INSENSITIVE);
		// cnn archive
		Pattern p = Pattern.compile("<div class=\"cnn_sclsts\">\\s*<a\\s+href\\s*=\\s*\"?(http)?(.*?)[\"|>]?(.*?)</a>", Pattern.CASE_INSENSITIVE);
		Matcher m = p.matcher(fullText);
		while(m.find())
		{
			String link = m.group().trim();
			String hrefLink=null;
			if(link!=null){
				hrefLink = link.substring(link.indexOf("http"),link.indexOf(".html")+5);
				links.add(hrefLink);
			}
			//System.out.println(link);
			//System.out.println(hrefLink);
		}
		return links;
	
	}
	
	private String downloadArticleBody(String content)
	{
		String fullBody = "";
		try
		{
			//CNN Article
			//Pattern p = Pattern.compile("<p.*</p>", Pattern.MULTILINE);
			//FT
			Pattern p = Pattern.compile("<div id=\"artpagination\">\\s*.*</div>", Pattern.MULTILINE);
			Matcher m = p.matcher(content);
			while(m.find())
			{
				String data = m.group().trim();
				if(data!=null)
					fullBody+=data;
				System.out.println(fullBody);
			}
        }
        catch (Exception ex)
        {
            ex.printStackTrace();
        }
        return fullBody;
	}

	public String downloadCNNArticleTitle(String content){
		String title = "";
		try
		{
			Pattern p = Pattern.compile("<title>.*</title>", Pattern.MULTILINE);
			Matcher m = p.matcher(content);
			while(m.find())
			{
				String data = m.group().trim();
				if(data!=null)
					title=data.substring(data.indexOf("<title>")+7,data.indexOf("</title>") );
					System.out.println(title);
			}
        }
        catch (Exception ex)
        {
            ex.printStackTrace();
        }
        return title;
	}
	
	
	
	
	private String downloadPage(String url)
	{
		String fullText = "";
		try
		{
            URL my_url = new URL(url);
            BufferedReader br = new BufferedReader(new InputStreamReader(my_url.openStream()));
            String text = "";
            while(null != (text = br.readLine()))
            {
            	fullText += text + "\n";
            }
        }
        catch (Exception ex)
        {
            ex.printStackTrace();
        }
		//System.out.println(fullText);
        return fullText;
        
	}
	


}
